package com.yc.service.impl;

import io.github.bonigarcia.wdm.WebDriverManager;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.TimeoutException;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.PageLoadStrategy;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.ExpectedCondition;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.springframework.stereotype.Service;

import com.yc.service.ProductCrawlerService;

import java.time.Duration;
import java.util.*;
import java.util.NoSuchElementException;

@Service
public class ProductCrawlerServiceImpl implements ProductCrawlerService {

    private static String collectedCookies = "";

    @Override
    public Map<String, Object> crawlProducts(String supplierUrl, String cookies) {
        List<String> logs = new ArrayList<>();
        Map<String, Object> resultMap = new HashMap<>();
        // 使用最新的ChromeDriver版本并清理缓存
        WebDriverManager.chromedriver().clearDriverCache().setup();

        try {
            // 1. 初始化WebDriver配置（带代理轮换重试）
            addLog(logs, "开始初始化WebDriver...");
            WebDriverManager.chromedriver().setup();
            addLog(logs, "WebDriverManager配置完成");

            int launchAttempts = Math.max(1, useProxy ? Math.min(3, PROXY_POOL.size()) : 1);
            addLog(logs, "浏览器启动尝试次数: " + launchAttempts);
            Exception lastLaunchException = null;
            for (int attempt = 0; attempt < launchAttempts; attempt++) {
                addLog(logs, "浏览器启动尝试 #" + (attempt + 1) + "/" + launchAttempts);
                try {
                    addLog(logs, "构建ChromeOptions配置...");
                    ChromeOptions options = getChromeOptions(useProxy, headless);
                    addLog(logs, "创建ChromeDriver实例...");
                    driver = new ChromeDriver(options);
                    driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(5));
                    addLog(logs, "浏览器启动成功");
                    break;
                } catch (Exception e) {
                    lastLaunchException = e;
                    addLog(logs, "浏览器启动失败(尝试 #" + (attempt + 1) + "): " + e.getMessage());
                    if (driver != null) {
                        try {
                            driver.quit();
                            addLog(logs, "浏览器实例已清理");
                        } catch (Exception ignored) {
                        }
                        driver = null;
                    }
                    // 小退避
                    long sleepTime = 1 + attempt;
                    addLog(logs, "等待 " + sleepTime + " 秒后重试...");
                    try {
                        TimeUnit.SECONDS.sleep(sleepTime);
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt();
                    }
                }
            }
            if (driver == null) {
                addLog(logs, "所有浏览器启动尝试失败，最终异常: " + (lastLaunchException != null ? lastLaunchException.getMessage() : "未知错误"));
                throw new RuntimeException("浏览器启动失败", lastLaunchException);
            }

            addLog(logs, "创建WebDriverWait实例，超时设置: " + Math.max(5, timeoutSec) + "秒");
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(Math.max(5, timeoutSec)));

            log.info("启动浏览器并访问: {}", supplierUrl);
            addLog(logs, "开始访问目标URL: " + supplierUrl);

            // 2. 访问目标页面（带重试机制）
            addLog(logs, "开始带重试机制的页面访问，最大重试次数: 3");
            boolean accessSuccess = accessWithRetry(driver, supplierUrl, 3, logs);
            if (!accessSuccess) {
                addLog(logs, "错误: 多次尝试后仍无法访问目标页面");
                throw new RuntimeException("多次尝试后仍无法访问目标页面");
            } else {
                addLog(logs, "页面访问成功，当前URL: " + driver.getCurrentUrl());
                addLog(logs, "页面标题: " + driver.getTitle());
            }

            // 3. Cookie管理 - 获取供应商标识
            addLog(logs, "开始Cookie管理流程");
            String supplierId = generateSupplierId(supplierName, supplierUrl);
            addLog(logs, "生成的供应商ID: " + supplierId);
            
            // 4. 尝试加载已保存的Cookie
            addLog(logs, "尝试加载已保存的Cookie...");
            String savedCookie = loadCookie(supplierId, logs);
            
            // 5. 优先使用提供的cookie，其次使用保存的cookie
            String cookieToUse = (cookies != null && !cookies.isEmpty()) ? cookies : savedCookie;
            addLog(logs, "Cookie来源: " + (cookies != null && !cookies.isEmpty() ? "用户提供的Cookie" : (savedCookie != null ? "已保存的Cookie" : "无Cookie可用")));
            
            if (cookieToUse != null && !cookieToUse.isEmpty()) {
                addLog(logs, "正在设置Cookie（" + (cookies != null && !cookies.isEmpty() ? "用户提供" : "已保存") + "）...");
                addLog(logs, "Cookie长度: " + cookieToUse.length() + " 字符");
                setCookies(driver, cookieToUse);
                // 设置cookie后刷新页面以应用登录状态
                addLog(logs, "Cookie设置完成，开始刷新页面以应用登录状态...");
                driver.navigate().refresh();
                addLog(logs, "页面刷新完成，验证Cookie是否生效");
            } else {
                addLog(logs, "未提供有效Cookie，将以未登录状态继续");
            }

            // 4. 处理可能的登录弹窗
            addLog(logs, "===== 开始处理登录状态 =====");
            handleLoginPopup(driver, wait, logs);
            addLog(logs, "===== 登录状态处理完成 =====");

            // 5. 智能滚动加载（优化版）
            addLog(logs, "===== 开始智能滚动加载 =====");
            addLog(logs, "最大滚动次数设置为: " + Math.max(1, maxScrolls));
            simulateSmartScrolling(driver, logs, Math.max(1, maxScrolls), config);
            addLog(logs, "===== 智能滚动加载完成 =====");

            // 6. 查找商品容器元素
            addLog(logs, "===== 开始查找商品容器元素 =====");
            By containerSelector = By.cssSelector("div.antd-pro-components-goods-list-components-item-index-container");
            addLog(logs, "使用特定选择器查找商品容器: " + containerSelector.toString());
            
            // 等待页面加载完成
            try {
                wait.until(ExpectedConditions.presenceOfElementLocated(containerSelector));
                addLog(logs, "确认页面中存在商品容器元素");
            } catch (Exception e) {
                addLog(logs, "等待商品容器超时，但继续尝试查找: " + e.getMessage());
            }
            
            // 找到所有商品容器
            List<WebElement> productContainers = driver.findElements(containerSelector);
            addLog(logs, "页面总商品数: " + productContainers.size());
            
            // 7. 提取商品数据
            addLog(logs, "===== 开始提取商品数据 =====");
            addLog(logs, "准备从 " + productContainers.size() + " 个商品容器中提取数据");
            
            int extractedCount = 0;
            int skippedCount = 0;
            
            for (int i = 0; i < productContainers.size(); i++) {
                WebElement container = productContainers.get(i);
                addLog(logs, "处理商品容器 #" + (i + 1) + "/" + productContainers.size());
                
                // 速率限制
                if (!config.skipElementDelay) {
                    try {
                        Thread.sleep(config.elementProcessDelay);
                        addLog(logs, "元素处理延迟：" + config.elementProcessDelay + "ms");
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt();
                        addLog(logs, "线程被中断");
                    }
                }
                
                // 从容器中提取商品数据
                Map<String, String> product = new HashMap<>();
                boolean hasValidData = false;
                
                // 主图提取
                try {
                    WebElement img = container.findElement(
                            By.cssSelector("img.antd-pro-components-goods-list-components-item-index-imgCover"));
                    String imgUrl = img.getAttribute("src");
                    if (imgUrl != null && imgUrl.startsWith("http")) {
                        product.put("image", imgUrl);
                        addLog(logs, "提取到主图: " + imgUrl);
                        hasValidData = true;
                    }
                } catch (org.openqa.selenium.NoSuchElementException ignored) {
                    addLog(logs, "未找到主图元素");
                }
                
                // 商品名称提取
                try {
                    WebElement nameElem = container.findElement(
                            By.cssSelector(".antd-pro-components-goods-list-components-item-index-itemTitle"));
                    String name = nameElem.getText().trim();
                    if (!name.isEmpty()) {
                        product.put("name", name);
                        addLog(logs, "提取到商品名称: " + name);
                        hasValidData = true;
                    }
                } catch (org.openqa.selenium.NoSuchElementException ignored) {
                    try {
                        WebElement miniName = container.findElement(
                                By.cssSelector(".antd-pro-components-goods-list-components-item-index-miniTitle"));
                        String name = miniName.getText().trim();
                        if (!name.isEmpty()) {
                            product.put("name", name);
                            addLog(logs, "提取到商品名称(mini): " + name);
                            hasValidData = true;
                        }
                    } catch (org.openqa.selenium.NoSuchElementException ignored2) {
                        addLog(logs, "未找到商品名称元素");
                    }
                }
                
                // 商品价格提取
                try {
                    WebElement priceElem = container.findElement(
                            By.cssSelector("[class*='price']"));
                    String price = priceElem.getText().trim();
                    if (!price.isEmpty()) {
                        // 价格格式化（提取数字）
                        price = price.replaceAll("[^0-9.,]+", "").replace(",", "");
                        product.put("price", price);
                        addLog(logs, "提取到价格: " + price);
                        hasValidData = true;
                    }
                } catch (org.openqa.selenium.NoSuchElementException ignored) {
                    addLog(logs, "未找到价格元素");
                }
                
                // 添加完整文本用于调试
                String fullText = container.getText().trim();
                if (fullText.isEmpty()) {
                    JavascriptExecutor js = (JavascriptExecutor) driver;
                    fullText = (String) js.executeScript("return arguments[0].textContent.trim();", container);
                }
                product.put("fullText", fullText);
                
                if (hasValidData) {
                    products.add(product);
                    extractedCount++;
                    addLog(logs, "成功提取商品数据，当前累计提取: " + extractedCount + " 个商品");
                } else {
                    skippedCount++;
                    addLog(logs, "商品数据无效，已跳过，当前累计跳过: " + skippedCount + " 个商品");
                }
            }
            
            addLog(logs, "商品数据提取完成 - 成功提取: " + extractedCount + " 个, 跳过: " + skippedCount + " 个");

            // 9. 获取当前浏览器中的cookie（用于后续爬取复用）
            addLog(logs, "===== 开始Cookie获取与保存 =====");
            addLog(logs, "获取当前浏览器中的Cookie用于后续爬取复用");
            String currentCookies = getCookiesAsString(driver);
            addLog(logs, "成功获取当前会话Cookie，长度: " + (currentCookies != null ? currentCookies.length() : 0) + " 字符");
            
            // 10. 检查登录状态并保存Cookie
            addLog(logs, "检查登录状态并保存Cookie");
            boolean isLoggedIn = checkLoginStatus(driver, logs);
            addLog(logs, "登录状态检查结果: " + (isLoggedIn ? "已登录" : "未登录"));
            
            if (isLoggedIn && currentCookies != null && !currentCookies.isEmpty()) {
                addLog(logs, "用户已登录且获取到有效Cookie，开始保存到文件");
                saveCookie(supplierId, currentCookies, logs);
            } else {
                if (!isLoggedIn) {
                    addLog(logs, "未登录状态，跳过Cookie保存");
                } else if (currentCookies == null || currentCookies.isEmpty()) {
                    addLog(logs, "Cookie为空，跳过保存");
                }
            }

            // 10. 结果封装
            addLog(logs, "===== 开始封装爬取结果 =====");
            addLog(logs, "最终爬取商品数量: " + products.size());
            String endTime = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
            addLog(logs, "===== 爬取任务结束 ===== " + endTime);
            
            result.put("success", true);
            result.put("supplierName", supplierName);
            result.put("count", products.size());
            result.put("products", products);
            result.put("logs", logs);
            result.put("cookies", currentCookies); // 返回cookie供前端保存
            result.put("startTime", startTime);
            result.put("endTime", endTime);

        } catch (Exception e) {
            String endTime = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME);
            addLog(logs, "===== 爬取任务异常结束 ===== " + endTime);
            log.error("爬取异常: ", e); // 打印完整堆栈
            addLog(logs, "爬取失败：" + e.getMessage());
            addLog(logs, "异常详情: " + e.toString());
            // 添加堆栈信息到日志
            StringWriter sw = new StringWriter();
            PrintWriter pw = new PrintWriter(sw);
            e.printStackTrace(pw);
            String stackTrace = sw.toString();
            // 只保存部分堆栈信息，避免日志过大
            if (stackTrace.length() > 500) {
                stackTrace = stackTrace.substring(0, 500) + "...";
            }
            addLog(logs, "异常堆栈: " + stackTrace);
            
            result.put("success", false);
            result.put("error", e.getMessage());
            result.put("logs", logs);
            result.put("startTime", startTime);
            result.put("endTime", endTime);
        } finally {
            addLog(logs, "开始清理资源");
            if (driver != null) {
                try {
                    addLog(logs, "正在关闭浏览器...");
                    driver.quit();
                    addLog(logs, "浏览器已成功关闭");
                } catch (Exception e) {
                    addLog(logs, "关闭浏览器失败：" + e.getMessage());
                }
            } else {
                addLog(logs, "浏览器实例为null，无需关闭");
            }
            addLog(logs, "资源清理完成");
        }

        return result;
    }

    @Override
    public Map<String, Object> crawlJushuitan(String cookies, String url, boolean useProxy, boolean headless, 
                                            int maxScrolls, int timeoutSec, int rateLimitMs) {
        // 聚水潭需要登录，建议通过参数传入有效的Cookie
        // 如果用户没有提供url，则使用默认的url
        String targetUrl = url != null && !url.isEmpty()
                ? url
                : "https://www.jushuitan.com/index.html#/trade/order/list";

        // 对于聚水潭，根据rateLimitMs判断使用什么性能模式
        String performanceMode = rateLimitMs <= 100 ? "fast" : 
                               (rateLimitMs >= 300 ? "stable" : "balanced");
        
        return crawlProducts("聚水潭供应商",
                targetUrl,
                cookies,
                useProxy, // 聚水潭反爬严格，建议启用代理
                headless,
                timeoutSec,
                maxScrolls,
                performanceMode);
    }

    @Override
    public Map<String, Object> getJushuitanTestInfo() {
        Map<String, Object> result = new HashMap<>();
        result.put("message", "聚水潭抓取测试接口");
        result.put("url", "https://www.jushuitan.com/index.html#/trade/order/list");
        result.put("recommended_params", Map.of(
                "headless", false, // 建议非无头模式便于调试
                "maxScrolls", 20, // 聚水潭页面内容较多，建议更多滚动
                "timeoutSec", 45, // 聚水潭加载较慢，建议更长超时
                "rateLimitMs", 800, // 聚水潭反爬严格，建议更慢速率
                "useProxy", true // 建议使用代理
        ));
        result.put("usage_example",
                "GET /api/crawler/crawl-jushuitan?cookies=your_cookies&headless=false&maxScrolls=20&timeoutSec=45&rateLimitMs=800");
        return result;
    }

    /**
     * 构建Chrome浏览器配置（优化连接稳定性）
     */
    private ChromeOptions getChromeOptions(boolean useProxy, boolean headless) {
        ChromeOptions options = new ChromeOptions();
        // 基础配置
        options.addArguments("--start-maximized");
        options.addArguments("--disable-blink-features=AutomationControlled");
        options.addArguments("--disable-notifications");
        // 解决WebSocket 403连接问题
        options.addArguments("--remote-allow-origins=*");
        options.addArguments("--remote-debugging-port=9222");
        // 优化性能和稳定性
        options.addArguments("--disable-gpu");
        options.addArguments("--disable-extensions");
        options.addArguments("--ignore-certificate-errors");
        options.addArguments("--disable-dev-shm-usage");
        options.addArguments("--no-sandbox");
        options.addArguments("--window-size=1920,1080");
        // 提高爬取稳定性
        options.setPageLoadStrategy(PageLoadStrategy.NORMAL);
        // 禁用自动化控制检测
        options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
        // 添加CDP版本兼容性处理参数
        options.addArguments("--disable-features=VizDisplayCompositor,IsolateOrigins,site-per-process");
        options.addArguments("--disable-features=NetworkService,NetworkServiceInProcess");
        options.addArguments("--disable-features=ScriptStreaming");
        // 设置更安全的WebSocket连接选项
        options.addArguments("--no-proxy-server");
        options.addArguments("--disable-features=BrowserSideNavigation");
        // 减少资源消耗
        options.addArguments("--disable-background-networking");
        options.addArguments("--disable-component-update");
        // 禁用崩溃报告
        options.addArguments("--disable-crash-reporter");
        options.addArguments("--disable-error-dialogs");
        // 添加额外的稳定性参数
        options.addArguments("--disable-features=FlashDeprecationWarning");
        options.addArguments("--disable-features=IsolateOrigins,site-per-process");
        options.addArguments("--disable-features=ImprovedCookieControls");
        options.addArguments("--disable-features=CookiesWithoutSameSiteMustBeSecure");
        options.addArguments("--disable-features=SameSiteByDefaultCookies");
        options.addArguments("--disable-features=NetworkPrediction");
        options.addArguments("--disable-features=AccountConsistency");
        options.addArguments("--disable-features=TranslateUI");
        options.addArguments("--disable-in-process-stack-traces");
        options.addArguments("--disable-logging");
        options.addArguments("--disable-permissions-api");
        // 禁用WebGL以提高兼容性
        options.addArguments("--disable-webgl");
        // 禁用GPU加速渲染
        options.addArguments("--disable-accelerated-2d-canvas");
        options.addArguments("--disable-accelerated-compositing");
        options.addArguments("--disable-accelerated-layers");
        options.addArguments("--disable-accelerated-plugins");
        // 禁用硬件加速
        options.addArguments("--disable-hardware-acceleration");
        // 设置更低的性能配置以提高稳定性
        options.addArguments("--low-end-device");
        // 禁用自动关闭功能
        options.setExperimentalOption("detach", true);

        WebDriver driver = new ChromeDriver(options);
        WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(25));

        try {
            // 访问页面
            driver.get(supplierUrl);
            logMessage("访问页面: " + supplierUrl, logs);

            // 设置Cookie
            if (cookies != null && !cookies.isEmpty()) {
                setCookies(driver, cookies, logs);
                driver.get(supplierUrl);
                logMessage("已应用提供的Cookie并重新加载页面", logs);
            } else {
                collectedCookies = collectCookies(driver);
                if (!collectedCookies.isEmpty()) {
                    logMessage("已自动收集Cookie", logs);
                }
            }

            waitForPageLoad(driver, logs);

            // 检测是否已登录，未登录则等待用户登录
            waitForLogin(driver, logs);

            List<WebElement> productContainers = driver.findElements(
                    By.cssSelector("div.antd-pro-components-goods-list-components-item-index-container"));
            logMessage("页面总商品数: " + productContainers.size(), logs);

            List<Map<String, String>> allProducts = new ArrayList<>();

            for (WebElement product : productContainers) {
                Map<String, String> info = new HashMap<>();

                // 主图
                try {
                    WebElement img = product.findElement(
                            By.cssSelector("img.antd-pro-components-goods-list-components-item-index-imgCover"));
                    String imgUrl = img.getAttribute("src");
                    if (imgUrl != null && imgUrl.startsWith("http")) {
                        info.put("image", imgUrl);
                    }
                } catch (NoSuchElementException ignored) {
                }

                // 商品名称
                try {
                    WebElement nameElem = product.findElement(
                            By.cssSelector(".antd-pro-components-goods-list-components-item-index-itemTitle"));
                    info.put("name", nameElem.getText().trim());
                } catch (NoSuchElementException e1) {
                    try {
                        WebElement miniName = product.findElement(
                                By.cssSelector(".antd-pro-components-goods-list-components-item-index-miniTitle"));
                        info.put("name", miniName.getText().trim());
                    } catch (NoSuchElementException ignored2) {
                    }
                }

                // 商品价格
                try {
                    WebElement priceElem = product.findElement(
                            By.cssSelector("[class*='price']"));
                    info.put("price", priceElem.getText().trim());
                } catch (NoSuchElementException ignored) {
                }

                allProducts.add(info);
            }

            resultMap.put("products", allProducts);
            resultMap.put("cookies", collectedCookies);
            resultMap.put("totalCount", allProducts.size());
            resultMap.put("logs", logs);

        } catch (Exception e) {
            logMessage("爬取过程中出错: " + e.getMessage(), logs);
            resultMap.put("products", new ArrayList<>());
            resultMap.put("cookies", collectedCookies);
            resultMap.put("error", e.getMessage());
            resultMap.put("logs", logs);
        } finally {
            try {
                // 延长延迟时间到20秒，确保浏览器有足够时间保持打开状态
                logMessage("等待浏览器操作完成...", logs);
                Thread.sleep(20000);
                logMessage("浏览器操作时间结束，准备关闭WebDriver", logs);
            } catch (InterruptedException ignored) {
            }
            // 注释掉driver.quit()以防止浏览器被关闭
            // driver.quit();
            logMessage("WebDriver会话保持活动状态", logs);
        }

        return resultMap;
    }

    private void logMessage(String message, List<String> logs) {
        // 添加到日志列表
        logs.add(message);
        // 同时输出到控制台
        System.out.println("[爬虫日志] " + message);
    }

    private void waitForPageLoad(WebDriver driver, List<String> logs) {
        try {
<<<<<<< HEAD
            Map<String, String> product = new HashMap<>();
            JavascriptExecutor js = (JavascriptExecutor) driver;
            
            // 获取元素基本信息
            String tagName = parent.getTagName();
            addLog(logs, "处理元素类型: " + tagName);
            
            // 获取元素HTML预览（用于调试）
            String htmlPreview = (String) js.executeScript("return arguments[0].outerHTML.substring(0, 200);", parent);
            addLog(logs, "元素HTML预览: " + htmlPreview + "...");
            
            // 判断元素类型（表格行或卡片）
            boolean isTableRow = tagName.equals("tr") || parent.getAttribute("class").contains("table-row");
            boolean isCard = parent.getAttribute("class").contains("card");
            addLog(logs, "元素类型判断: 表格行=" + isTableRow + ", 卡片=" + isCard);
            
            // 提取商品名称
            product.put("name", extractTextWithFallback(parent, Arrays.asList(
                    ".product-name, .goods-name, .item-title",
                    ".ant-card-title, .ant-card-head-title",
                    ".ant-table-cell:nth-child(2), .table-cell:nth-child(2)",
                    ".name, .title",
                    "[class*='name'], [class*='title']",
                    ".text, .content",
                    "h3, h4, h5, h6",
                    "a[href]",
                    ".ant-descriptions-item-content",
                    "[data-title], [aria-label]",
                    ".ant-typography",
                    ".info .title",
                    ".detail-name",
                    ".product-info .name",
                    "[class*='businessDynamics']",
                    ".ant-list-item-meta-title"), js));
            
            if (product.get("name") != null) {
                addLog(logs, "提取到商品名称: " + product.get("name"));
            }

            // 提取价格
            String price = extractTextWithFallback(parent, Arrays.asList(
                    ".product-price, .goods-price, .price",
                    ".ant-table-cell:nth-child(3), .table-cell:nth-child(3)",
                    ".money, .amount, .value",
                    "span[class*='price'], div[class*='price']",
                    "[class*='amount'], .cost",
                    ".ant-statistic-content",
                    ".ant-typography-price",
                    "[data-price]",
                    "//span[contains(text(),'¥') or contains(text(),'$')]",
                    ".price-info .value"), js);
            
            // 价格格式化（提取数字）
            if (price != null) {
                price = price.replaceAll("[^0-9.,]+", "").replace(",", "");
                addLog(logs, "提取到价格: " + price);
            }
            product.put("price", price);

            // 提取图片 - 使用特定选择器爬取主图
            try {
                WebElement img = parent.findElement(
                        By.cssSelector("img.antd-pro-components-goods-list-components-item-index-imgCover"));
                String imgUrl = img.getAttribute("src");
                if (imgUrl != null && imgUrl.startsWith("http")) {
                    product.put("image", imgUrl);
                    addLog(logs, "成功使用特定选择器提取主图: " + imgUrl);
                }
            } catch (Exception e) {
                // 如果特定选择器失败，使用回退方法
                addLog(logs, "特定选择器提取主图失败，使用回退方法: " + e.getMessage());
                String fallbackImg = extractAttrWithFallback(parent, "img", Arrays.asList(
                        "src", "data-src", "data-original", "data-img"), js);
                if (fallbackImg != null) {
                    product.put("image", fallbackImg);
                    addLog(logs, "回退方法提取到图片: " + fallbackImg);
                }
            }

            // 提取链接
            String url = extractAttrWithFallback(parent, "a[href]", Arrays.asList("href"), js);
            if (url != null && !url.startsWith("http")) {
                // 转换相对路径为绝对路径
                String baseUrl = driver.getCurrentUrl();
                if (url.startsWith("/")) {
                    url = baseUrl.split("//")[0] + "//" + baseUrl.split("//")[1].split("/")[0] + url;
                } else {
                    url = baseUrl.endsWith("/") ? baseUrl + url : baseUrl + "/" + url;
                }
            }
            product.put("url", url);

            // 提取销量/数量
            product.put("sales", extractTextWithFallback(parent, Arrays.asList(
                    ".ant-table-cell:nth-child(4), .table-cell:nth-child(4)",
                    ".sales, .sell-count, .count, .quantity",
                    "[class*='sales'], [class*='count']",
                    ".ant-badge-count",
                    ".number",
                    "[data-count]"), js));

            // 提取SKU/ID
            product.put("sku", extractTextWithFallback(parent, Arrays.asList(
                    ".ant-table-cell:nth-child(1), .table-cell:nth-child(1)",
                    ".sku, .code, .id",
                    "[class*='sku'], [class*='code'], [class*='id']",
                    ".ant-descriptions-item-label:contains('SKU') + .ant-descriptions-item-content",
                    "[data-sku]",
                    ".product-code"), js));

            // 提取状态信息
            product.put("status", extractTextWithFallback(parent, Arrays.asList(
                    ".ant-table-cell:nth-child(5), .table-cell:nth-child(5)",
                    ".status, .state",
                    "[class*='status'], [class*='state']",
                    ".ant-tag",
                    ".status-badge",
                    ".tag",
                    ".ant-badge"), js));

            // 提取时间信息
            product.put("time", extractTextWithFallback(parent, Arrays.asList(
                    ".ant-table-cell:nth-child(6), .table-cell:nth-child(6)",
                    ".time, .date",
                    "[class*='time'], [class*='date']",
                    ".ant-typography-text",
                    ".timestamp",
                    "[data-time]",
                    ".update-time"), js));
            
            // 获取完整文本内容
            String fullText = parent.getText().trim();
            if (fullText.isEmpty()) {
                // 使用JS作为备选获取文本
                fullText = (String) js.executeScript("return arguments[0].textContent.trim();", parent);
            }
            addLog(logs, "元素完整文本长度: " + fullText.length() + " 字符");
            product.put("fullText", fullText); // 保存完整文本
            
            // 验证提取的数据
            boolean hasAnyValue = false;
            StringBuilder extractedFields = new StringBuilder();
            
            for (Map.Entry<String, String> entry : product.entrySet()) {
                if (entry.getValue() != null && !entry.getValue().trim().isEmpty()) {
                    hasAnyValue = true;
                    extractedFields.append(entry.getKey()).append(": '").append(entry.getValue().substring(0, Math.min(30, entry.getValue().length()))).append("' ");
                }
            }
            
            if (hasAnyValue) {
                addLog(logs, "成功提取商品数据: " + extractedFields.toString());
                return product;
            } else {
                addLog(logs, "商品元素无有效数据，跳过");
                return null;
            }
=======
            logMessage("等待页面加载完成...", logs);
            new WebDriverWait(driver, Duration.ofSeconds(30)).until(
                    webDriver -> ((JavascriptExecutor) webDriver).executeScript("return document.readyState")
                            .equals("complete"));
            Thread.sleep(2000);
            logMessage("页面加载完成", logs);
>>>>>>> 92930f614e336ec41682689003b3115fb39b0efc
        } catch (Exception e) {
            logMessage("等待页面加载超时: " + e.getMessage(), logs);
        }
    }

    private void setCookies(WebDriver driver, String cookies, List<String> logs) {
        try {
            logs.add("设置Cookie...");
            String[] cookiePairs = cookies.split("; ");
            for (String cookiePair : cookiePairs) {
                if (cookiePair.contains("=")) {
                    String[] parts = cookiePair.split("=", 2);
                    if (parts.length == 2) {
                        Cookie cookie = new Cookie.Builder(parts[0].trim(), parts[1].trim())
                                .domain(driver.getCurrentUrl().split("/")[2])
                                .path("/")
                                .build();
                        driver.manage().addCookie(cookie);
                    }
                }
            }
            logs.add("Cookie设置完成");
        } catch (Exception e) {
            logs.add("设置Cookie失败: " + e.getMessage());
        }
    }

    private String collectCookies(WebDriver driver) {
        try {
            Set<Cookie> cookies = driver.manage().getCookies();
            StringBuilder sb = new StringBuilder();
            for (Cookie c : cookies) {
                sb.append(c.getName()).append("=").append(c.getValue()).append("; ");
            }
            String result = sb.toString();
            return result.isEmpty() ? "" : result.substring(0, result.length() - 2);
        } catch (Exception e) {
            return "";
        }
    }

    /**
     * 等待用户登录完成
     * 检测页面上是否有登录相关的元素来判断登录状态
     */
    private void waitForLogin(WebDriver driver, List<String> logs) {
        logMessage("开始检测登录状态...", logs);

        try {
            // 定义一些常见的登录成功标志元素选择器
            // 移除过于宽泛的username选择器，避免匹配登录框
            String[] loggedInSelectors = {
                    ".user-profile:not(input):not(select):not(textarea)", // 用户信息区域(排除表单元素)
                    ".avatar:not(input):not(select):not(textarea)", // 头像元素(排除表单元素)
                    "[class*='user-info']:not(input):not(select):not(textarea)", // 用户信息(排除表单元素)
                    "[class*='nav-user']:not(input):not(select):not(textarea)" // 导航用户(排除表单元素)
            };

            // 定义一些常见的登录表单元素选择器
            String[] loginFormSelectors = {
                    "[id*='login']", // 包含login的id
                    "[name*='password']", // 密码输入框
                    "[type='submit'][value*='登录']", // 登录按钮
                    ".login-form", // 登录表单
                    "[class*='login-']" // 包含login-的class
            };

            // 检查是否已经登录（查找登录成功标志）
            boolean isLoggedIn = false;
            for (String selector : loggedInSelectors) {
                try {
                    List<WebElement> elements = driver.findElements(By.cssSelector(selector));
                    // 不仅检查元素存在，还要检查元素是否可见且不为空
                    for (WebElement element : elements) {
                        if (element.isDisplayed() && !element.getTagName().matches("input|select|textarea")) {
                            isLoggedIn = true;
                            logMessage("检测到登录成功标志: " + selector + " (可见且非表单元素)", logs);
                            break;
                        }
                    }
                    if (isLoggedIn)
                        break;
                } catch (Exception ignored) {
                    // 忽略选择器错误
                }
            }

            // 如果已经登录，给页面更多时间加载商品数据
            if (isLoggedIn) {
                logMessage("用户已登录，给页面更多时间加载商品数据...", logs);
                // 增加额外的等待时间确保商品数据加载完成
                Thread.sleep(3000);
                // 等待可能的AJAX请求完成
                waitForAjaxCompletion(driver, logs);
                return;
            }

            // 检查是否有登录表单（需要用户手动登录）
            boolean hasLoginForm = false;
            for (String selector : loginFormSelectors) {
                try {
                    List<WebElement> elements = driver.findElements(By.cssSelector(selector));
                    if (!elements.isEmpty()) {
                        hasLoginForm = true;
                        logMessage("检测到登录表单元素: " + selector, logs);
                        break;
                    }
                } catch (Exception ignored) {
                    // 忽略选择器错误
                }
            }

            // 如果检测到登录表单，提示用户登录
            if (hasLoginForm) {
                logMessage("请在浏览器中手动登录系统...", logs);
                logMessage("登录完成后，请按Enter键继续或等待自动检测", logs);

                // 等待用户登录完成，每5秒检查一次登录状态，最长等待2分钟
                long startTime = System.currentTimeMillis();
                long maxWaitTime = 2 * 60 * 1000; // 2分钟

                while (System.currentTimeMillis() - startTime < maxWaitTime) {
                    // 刷新页面，确保能获取最新状态
                    driver.navigate().refresh();
                    waitForPageLoad(driver, logs);

                    // 再次检查登录状态
                    isLoggedIn = false;
                    for (String selector : loggedInSelectors) {
                        try {
                            List<WebElement> elements = driver.findElements(By.cssSelector(selector));
                            if (!elements.isEmpty()) {
                                isLoggedIn = true;
                                logMessage("登录成功！检测到用户已登录", logs);
                                // 重新收集Cookie
                                collectedCookies = collectCookies(driver);
                                logMessage("已更新登录后的Cookie", logs);

                                // 登录成功后，给页面充分时间加载商品数据
                                logMessage("登录成功，等待商品数据加载...", logs);
                                Thread.sleep(5000); // 延长等待时间
                                waitForAjaxCompletion(driver, logs);
                                break;
                            }
                        } catch (Exception ignored) {
                            // 忽略选择器错误
                        }
                    }

                    if (isLoggedIn) {
                        break;
                    }

                    // 等待5秒后再次检查
                    logMessage("未检测到登录，请继续在浏览器中登录...", logs);
                    Thread.sleep(5000);
                }

                if (!isLoggedIn) {
                    logMessage("登录等待超时，将尝试继续爬取", logs);
                }
            } else {
                // 如果没有检测到明显的登录表单或登录状态，可能是不需要登录或者选择器不匹配
                logMessage("未检测到明确的登录表单或登录状态，将尝试继续爬取", logs);
            }

        } catch (Exception e) {
            logMessage("登录检测过程中出错: " + e.getMessage(), logs);
            logMessage("将尝试继续爬取", logs);
        }
    }

    /**
     * 等待AJAX请求完成
     * 用于确保页面上的异步数据加载完成
     */
    private void waitForAjaxCompletion(WebDriver driver, List<String> logs) {
        logMessage("等待AJAX请求完成...", logs);
        try {
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10));
            wait.until((ExpectedCondition<Boolean>) wd -> {
                try {
                    Boolean isAjaxComplete = (Boolean) ((JavascriptExecutor) wd).executeScript(
                            "return (window.jQuery != undefined) ? jQuery.active == 0 : true");
                    Boolean isDocumentReady = (Boolean) ((JavascriptExecutor) wd).executeScript(
                            "return document.readyState == 'complete'");
                    return isAjaxComplete && isDocumentReady;
                } catch (Exception e) {
                    return true; // 如果执行出错，默认认为已完成
                }
            });
            logMessage("AJAX请求完成", logs);
        } catch (Exception e) {
            logMessage("等待AJAX完成时出错: " + e.getMessage(), logs);
            logMessage("继续执行爬取操作", logs);
        }
    }
}
